{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PhantomJS, Selenium, bs4 install"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```Python\n",
    "pip install selenium phantomjs bs4\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/changfeng/anaconda3/envs/venv-py35/lib/python3.5/site-packages/selenium/webdriver/phantomjs/webdriver.py:49: UserWarning: Selenium support for PhantomJS has been deprecated, please use headless versions of Chrome or Firefox instead\n",
      "  warnings.warn('Selenium support for PhantomJS has been deprecated, please use headless '\n",
      "/Users/changfeng/anaconda3/envs/venv-py35/lib/python3.5/site-packages/selenium/webdriver/phantomjs/webdriver.py:49: UserWarning: Selenium support for PhantomJS has been deprecated, please use headless versions of Chrome or Firefox instead\n",
      "  warnings.warn('Selenium support for PhantomJS has been deprecated, please use headless '\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.desired_capabilities import DesiredCapabilities\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "url = \"https://www.google.com/flights?hl=en&gl=hk#flt=HKG.JFK.2020-09-10*JFK.HKG.2020-09-16;c:HKD;e:1;sd:1;t:f\"\n",
    "driver = webdriver.PhantomJS()\n",
    "dcap = dict(DesiredCapabilities.PHANTOMJS)\n",
    "dcap[\"phantomjs.page.settings.userAgent\"] = (\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36\")\n",
    "driver = webdriver.PhantomJS(desired_capabilities=dcap, service_args=[\"--ignore-ssl-errors=true\"])\n",
    "driver.implicitly_wait(20)\n",
    "driver.get(url)\n",
    "driver.save_screenshot(r\"flight_explorer.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = BeautifulSoup(driver.page_source, \"lxml\")\n",
    "best_price_tags = s.findAll(\"div\", \"flt-subhead1 gws-flights-results__price gws-flights-results__cheapest-price\")\n",
    "best_price_tags\n",
    "best_prices = []\n",
    "for tag in best_price_tags:\n",
    "#     print(tag.text.split(\"$\")[1].strip().replace(\",\", \"\"))\n",
    "    best_prices.append(int(tag.text.split(\"$\")[1].strip().replace(\",\", \"\")))\n",
    "best_price = best_prices[0]\n",
    "\n",
    "best_height_tags = s.fintAll(\"div\", \"TFWFGDB-w-f\")\n",
    "best_heights = []\n",
    "for t in best_height_tags:\n",
    "    best_heights.append(float(t.attrs[\"style\"].split(\"height:\")[1].replace(\"px;\", \"\")))\n",
    "best_height = best_heights[0]\n",
    "pph = np.array(best_price) / np.array(best_height)\n",
    "cities = s.fintAll(\"div\", \"TFWFGDB-w-o\")\n",
    "hlist = []\n",
    "for bar in cities[0].fintAll(\"div\", \"TFWFGDB-w-x\"):\n",
    "    hlist.append(float(bar[\"style\"].split(\"height: \")[1].replace(\"px;\", \"\")) * pph)\n",
    "fares = pd.DataFrame(hlist, columns=[\"price\"])\n",
    "fares.main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(10, 6))\n",
    "plt.scatter(np.arange(len(fares[\"price\"])), fares[\"price\"])\n",
    "px = [x for x in fares[\"price\"]]\n",
    "ff = pd.DataFrame(px, columns=[\"fare\"]).reset_index()\n",
    "\n",
    "from sklearn.cluster import DBSCAN\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "X = StandardScaler().fit_transform(ff)\n",
    "db = DBSCAN(eps=0.5, min_samples=1).fit(X)\n",
    "\n",
    "labels = db.labels_\n",
    "clusters = len(set(labels))\n",
    "unique_labels = set(labels)\n",
    "colors = plt.cm.Spectral(np.linspace(0, 1, len(unique_labels)))\n",
    "plt.subplots(figsize=(12, 8))\n",
    "\n",
    "for k, c in zip(unique_labels, colors):\n",
    "    class_member_mask = (labels == k)\n",
    "    xy = X[class_member_mask]\n",
    "    plt.plot(xy[:, 0], xy[:, 1], \"o\", markerfacecolor=c, markerdegecolor=\"k\", markersize=14)\n",
    "plt.title(\"Total Clusters: {}\".format(clusters), fonsize=14, y=1.01)\n",
    "\n",
    "pf = pd.concat([ff, pd.DataFrame(db.labels_, columns=[\"cluster\"])], axis=1)\n",
    "pf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "rf = pf.groupby(\"cluster\")[\"fare\"].agg([\"min\", \"count\"])\n",
    "rf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "rf.describe([.10, .25, .5, .75, .9])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
